from pinhole.datasource.spider import PinholeNativeSpider
from pinhole.datasource.document import Document

from scrapy.selector import Selector  # type: ignore
from markdownify import markdownify as md  # type: ignore
from datetime import datetime

import feedparser  # type: ignore
import dateparser


class GoogleProjectZeroBlog(PinholeNativeSpider):

    FEED_URL = "https://googleprojectzero.blogspot.com/feeds/posts/default"

    def __init__(self) -> None:
        super().__init__()

    def run(self, temp_file: str, log_file: str) -> None:
        d = feedparser.parse(self.FEED_URL)
        for entry in d.entries:
            title = entry.title
            date = dateparser.parse(entry.published) or datetime.today()
            url = entry.link

            content = ""
            for feed_content in entry.content:
                if feed_content.type == 'text/html':
                    content += md(feed_content.value)

            self.result_queue.put(Document.build(
                title, date, url, 'Google Project Zero',
                content
            ))
